//===----------------------------------------------------------------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2023 - 2025 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//


// #############################################################################
// #                                                                           #
// #            DO NOT EDIT THIS FILE; IT IS AUTOGENERATED.                    #
// #                                                                           #
// #############################################################################


import Builtin

@_alwaysEmitIntoClient
@_transparent
internal func _atomicMemoryFence(
  ordering: AtomicUpdateOrdering
) {
  switch ordering {
    case .relaxed:
      break
    case .acquiring:
      Builtin.fence_acquire()
    case .releasing:
      Builtin.fence_release()
    case .acquiringAndReleasing:
      Builtin.fence_acqrel()
    case .sequentiallyConsistent:
      Builtin.fence_seqcst()
    default:
      fatalError("Unsupported ordering")
  }
}

@usableFromInline
@frozen
@_alignment(1)
internal struct _AtomicInt8Storage {
  @usableFromInline
  internal var _value: Builtin.Int8

  @_alwaysEmitIntoClient @_transparent
  internal init(_ value: Builtin.Int8) {
    self._value = value
  }
}

extension UnsafeMutablePointer where Pointee == _AtomicInt8Storage {
  /// Atomically loads a word starting at this address with the specified
  /// memory ordering.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicLoad(ordering: AtomicLoadOrdering) -> _AtomicInt8Storage {
    switch ordering {
    case .relaxed:
      return _AtomicInt8Storage(Builtin.atomicload_monotonic_Int8(_rawValue))
    case .acquiring:
      return _AtomicInt8Storage(Builtin.atomicload_acquire_Int8(_rawValue))
    case .sequentiallyConsistent:
      return _AtomicInt8Storage(Builtin.atomicload_seqcst_Int8(_rawValue))
    default:
      fatalError("Unsupported ordering")
    }
  }

  /// Atomically stores the specified value starting at the memory referenced by
  /// this pointer, with the specified memory ordering.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicStore(
    _ desired: _AtomicInt8Storage,
    ordering: AtomicStoreOrdering
  ) {
    switch ordering {
    case .relaxed:
      Builtin.atomicstore_monotonic_Int8(_rawValue, desired._value)
    case .releasing:
      Builtin.atomicstore_release_Int8(_rawValue, desired._value)
    case .sequentiallyConsistent:
      Builtin.atomicstore_seqcst_Int8(_rawValue, desired._value)
    default:
      fatalError("Unsupported ordering")
    }
  }

  /// Atomically stores the specified value starting at the memory referenced by
  /// this pointer, with the specified memory ordering.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicExchange(
    _ desired: _AtomicInt8Storage,
    ordering: AtomicUpdateOrdering
  ) -> _AtomicInt8Storage {
    switch ordering {
    case .relaxed:
      let oldValue = Builtin.atomicrmw_xchg_monotonic_Int8(
        _rawValue, desired._value)
      return _AtomicInt8Storage(oldValue)
    case .acquiring:
      let oldValue = Builtin.atomicrmw_xchg_acquire_Int8(
        _rawValue, desired._value)
      return _AtomicInt8Storage(oldValue)
    case .releasing:
      let oldValue = Builtin.atomicrmw_xchg_release_Int8(
        _rawValue, desired._value)
      return _AtomicInt8Storage(oldValue)
    case .acquiringAndReleasing:
      let oldValue = Builtin.atomicrmw_xchg_acqrel_Int8(
        _rawValue, desired._value)
      return _AtomicInt8Storage(oldValue)
    case .sequentiallyConsistent:
      let oldValue = Builtin.atomicrmw_xchg_seqcst_Int8(
        _rawValue, desired._value)
      return _AtomicInt8Storage(oldValue)
    default:
      fatalError("Unsupported ordering")
    }
  }

  /// Perform an atomic compare and exchange operation with the specified memory
  /// ordering.
  ///
  /// This operation is equivalent to the following pseudocode:
  ///
  /// ```
  /// atomic(self, ordering) { currentValue in
  ///   let original = currentValue
  ///   guard original == expected else { return (false, original) }
  ///   currentValue = desired
  ///   return (true, original)
  /// }
  /// ```
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicCompareExchange(
    expected: _AtomicInt8Storage,
    desired: _AtomicInt8Storage,
    ordering: AtomicUpdateOrdering
  ) -> (exchanged: Bool, original: _AtomicInt8Storage) {
    switch ordering {
    case .relaxed:
      let (oldValue, won) = Builtin.cmpxchg_monotonic_monotonic_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    case .acquiring:
      let (oldValue, won) = Builtin.cmpxchg_acquire_acquire_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    case .releasing:
      let (oldValue, won) = Builtin.cmpxchg_release_monotonic_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    case .acquiringAndReleasing:
      let (oldValue, won) = Builtin.cmpxchg_acqrel_acquire_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    case .sequentiallyConsistent:
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    default:
      fatalError("Unsupported ordering")
    }
  }

  /// Perform an atomic compare and exchange operation with the specified
  /// success/failure memory orderings.
  ///
  /// This operation is equivalent to the following pseudocode:
  ///
  /// ```
  /// atomic(self, ordering, failureOrdering) { currentValue in
  ///   let original = currentValue
  ///   guard original == expected else { return (false, original) }
  ///   currentValue = desired
  ///   return (true, original)
  /// }
  /// ```
  ///
  /// The `ordering` argument specifies the memory ordering to use when the
  /// operation manages to update the current value, while `failureOrdering`
  /// will be used when the operation leaves the value intact.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicCompareExchange(
    expected: _AtomicInt8Storage,
    desired: _AtomicInt8Storage,
    successOrdering: AtomicUpdateOrdering,
    failureOrdering: AtomicLoadOrdering
  ) -> (exchanged: Bool, original: _AtomicInt8Storage) {
    // FIXME: LLVM doesn't support arbitrary ordering combinations
    // yet, so upgrade the success ordering when necessary so that it
    // is at least as "strong" as the failure case.
    switch (successOrdering, failureOrdering) {
    case (.relaxed, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_monotonic_monotonic_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    case (.relaxed, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acquire_acquire_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    case (.relaxed, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    case (.acquiring, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_acquire_monotonic_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    case (.acquiring, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acquire_acquire_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    case (.acquiring, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    case (.releasing, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_release_monotonic_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    case (.releasing, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acqrel_acquire_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    case (.releasing, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    case (.acquiringAndReleasing, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_acqrel_monotonic_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    case (.acquiringAndReleasing, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acqrel_acquire_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    case (.acquiringAndReleasing, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    case (.sequentiallyConsistent, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_monotonic_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    case (.sequentiallyConsistent, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_acquire_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    case (.sequentiallyConsistent, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    default:
      preconditionFailure("Unsupported orderings")
    }
  }

  /// Perform an atomic compare and exchange operation with the specified
  /// success/failure memory orderings.
  ///
  /// This operation is equivalent to the following pseudocode:
  ///
  /// ```
  /// atomic(self, ordering, failureOrdering) { currentValue in
  ///   let original = currentValue
  ///   guard original == expected else { return (false, original) }
  ///   currentValue = desired
  ///   return (true, original)
  /// }
  /// ```
  ///
  /// The `ordering` argument specifies the memory ordering to use when the
  /// operation manages to update the current value, while `failureOrdering`
  /// will be used when the operation leaves the value intact.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicWeakCompareExchange(
    expected: _AtomicInt8Storage,
    desired: _AtomicInt8Storage,
    successOrdering: AtomicUpdateOrdering,
    failureOrdering: AtomicLoadOrdering
  ) -> (exchanged: Bool, original: _AtomicInt8Storage) {
    // FIXME: LLVM doesn't support arbitrary ordering combinations
    // yet, so upgrade the success ordering when necessary so that it
    // is at least as "strong" as the failure case.
    switch (successOrdering, failureOrdering) {
    case (.relaxed, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_monotonic_monotonic_weak_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    case (.relaxed, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acquire_acquire_weak_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    case (.relaxed, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_weak_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    case (.acquiring, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_acquire_monotonic_weak_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    case (.acquiring, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acquire_acquire_weak_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    case (.acquiring, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_weak_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    case (.releasing, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_release_monotonic_weak_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    case (.releasing, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acqrel_acquire_weak_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    case (.releasing, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_weak_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    case (.acquiringAndReleasing, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_acqrel_monotonic_weak_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    case (.acquiringAndReleasing, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acqrel_acquire_weak_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    case (.acquiringAndReleasing, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_weak_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    case (.sequentiallyConsistent, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_monotonic_weak_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    case (.sequentiallyConsistent, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_acquire_weak_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    case (.sequentiallyConsistent, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_weak_Int8(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt8Storage(oldValue))
    default:
      preconditionFailure("Unsupported orderings")
    }
  }

  /// Perform an atomic wrapping add operation and return the new value,
  /// with the specified memory ordering.
  ///
  /// - Note: This operation silently wraps around on overflow, like the
  /// `&+` operator does on `UInt` values.
  ///
  /// - Returns: The original value before the operation.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicLoadThenWrappingIncrement(
    by operand: _AtomicInt8Storage,
    ordering: AtomicUpdateOrdering
  ) -> _AtomicInt8Storage {
    switch ordering {
    case .relaxed:
      let value = Builtin.atomicrmw_add_monotonic_Int8(
        _rawValue, operand._value)
      return _AtomicInt8Storage(value)
    case .acquiring:
      let value = Builtin.atomicrmw_add_acquire_Int8(
        _rawValue, operand._value)
      return _AtomicInt8Storage(value)
    case .releasing:
      let value = Builtin.atomicrmw_add_release_Int8(
        _rawValue, operand._value)
      return _AtomicInt8Storage(value)
    case .acquiringAndReleasing:
      let value = Builtin.atomicrmw_add_acqrel_Int8(
        _rawValue, operand._value)
      return _AtomicInt8Storage(value)
    case .sequentiallyConsistent:
      let value = Builtin.atomicrmw_add_seqcst_Int8(
        _rawValue, operand._value)
      return _AtomicInt8Storage(value)
    default:
      preconditionFailure("Unsupported ordering")
    }
  }

  /// Perform an atomic wrapping subtract operation and return the new value,
  /// with the specified memory ordering.
  ///
  /// - Note: This operation silently wraps around on overflow, like the
  /// `&-` operator does on `UInt` values.
  ///
  /// - Returns: The original value before the operation.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicLoadThenWrappingDecrement(
    by operand: _AtomicInt8Storage,
    ordering: AtomicUpdateOrdering
  ) -> _AtomicInt8Storage {
    switch ordering {
    case .relaxed:
      let value = Builtin.atomicrmw_sub_monotonic_Int8(
        _rawValue, operand._value)
      return _AtomicInt8Storage(value)
    case .acquiring:
      let value = Builtin.atomicrmw_sub_acquire_Int8(
        _rawValue, operand._value)
      return _AtomicInt8Storage(value)
    case .releasing:
      let value = Builtin.atomicrmw_sub_release_Int8(
        _rawValue, operand._value)
      return _AtomicInt8Storage(value)
    case .acquiringAndReleasing:
      let value = Builtin.atomicrmw_sub_acqrel_Int8(
        _rawValue, operand._value)
      return _AtomicInt8Storage(value)
    case .sequentiallyConsistent:
      let value = Builtin.atomicrmw_sub_seqcst_Int8(
        _rawValue, operand._value)
      return _AtomicInt8Storage(value)
    default:
      preconditionFailure("Unsupported ordering")
    }
  }

  /// Perform an atomic bitwise AND operation and return the new value,
  /// with the specified memory ordering.
  ///
  /// - Returns: The original value before the operation.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicLoadThenBitwiseAnd(
    with operand: _AtomicInt8Storage,
    ordering: AtomicUpdateOrdering
  ) -> _AtomicInt8Storage {
    switch ordering {
    case .relaxed:
      let value = Builtin.atomicrmw_and_monotonic_Int8(
        _rawValue, operand._value)
      return _AtomicInt8Storage(value)
    case .acquiring:
      let value = Builtin.atomicrmw_and_acquire_Int8(
        _rawValue, operand._value)
      return _AtomicInt8Storage(value)
    case .releasing:
      let value = Builtin.atomicrmw_and_release_Int8(
        _rawValue, operand._value)
      return _AtomicInt8Storage(value)
    case .acquiringAndReleasing:
      let value = Builtin.atomicrmw_and_acqrel_Int8(
        _rawValue, operand._value)
      return _AtomicInt8Storage(value)
    case .sequentiallyConsistent:
      let value = Builtin.atomicrmw_and_seqcst_Int8(
        _rawValue, operand._value)
      return _AtomicInt8Storage(value)
    default:
      preconditionFailure("Unsupported ordering")
    }
  }

  /// Perform an atomic bitwise OR operation and return the new value,
  /// with the specified memory ordering.
  ///
  /// - Returns: The original value before the operation.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicLoadThenBitwiseOr(
    with operand: _AtomicInt8Storage,
    ordering: AtomicUpdateOrdering
  ) -> _AtomicInt8Storage {
    switch ordering {
    case .relaxed:
      let value = Builtin.atomicrmw_or_monotonic_Int8(
        _rawValue, operand._value)
      return _AtomicInt8Storage(value)
    case .acquiring:
      let value = Builtin.atomicrmw_or_acquire_Int8(
        _rawValue, operand._value)
      return _AtomicInt8Storage(value)
    case .releasing:
      let value = Builtin.atomicrmw_or_release_Int8(
        _rawValue, operand._value)
      return _AtomicInt8Storage(value)
    case .acquiringAndReleasing:
      let value = Builtin.atomicrmw_or_acqrel_Int8(
        _rawValue, operand._value)
      return _AtomicInt8Storage(value)
    case .sequentiallyConsistent:
      let value = Builtin.atomicrmw_or_seqcst_Int8(
        _rawValue, operand._value)
      return _AtomicInt8Storage(value)
    default:
      preconditionFailure("Unsupported ordering")
    }
  }

  /// Perform an atomic bitwise XOR operation and return the new value,
  /// with the specified memory ordering.
  ///
  /// - Returns: The original value before the operation.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicLoadThenBitwiseXor(
    with operand: _AtomicInt8Storage,
    ordering: AtomicUpdateOrdering
  ) -> _AtomicInt8Storage {
    switch ordering {
    case .relaxed:
      let value = Builtin.atomicrmw_xor_monotonic_Int8(
        _rawValue, operand._value)
      return _AtomicInt8Storage(value)
    case .acquiring:
      let value = Builtin.atomicrmw_xor_acquire_Int8(
        _rawValue, operand._value)
      return _AtomicInt8Storage(value)
    case .releasing:
      let value = Builtin.atomicrmw_xor_release_Int8(
        _rawValue, operand._value)
      return _AtomicInt8Storage(value)
    case .acquiringAndReleasing:
      let value = Builtin.atomicrmw_xor_acqrel_Int8(
        _rawValue, operand._value)
      return _AtomicInt8Storage(value)
    case .sequentiallyConsistent:
      let value = Builtin.atomicrmw_xor_seqcst_Int8(
        _rawValue, operand._value)
      return _AtomicInt8Storage(value)
    default:
      preconditionFailure("Unsupported ordering")
    }
  }
}

@usableFromInline
@frozen
@_alignment(2)
internal struct _AtomicInt16Storage {
  @usableFromInline
  internal var _value: Builtin.Int16

  @_alwaysEmitIntoClient @_transparent
  internal init(_ value: Builtin.Int16) {
    self._value = value
  }
}

extension UnsafeMutablePointer where Pointee == _AtomicInt16Storage {
  /// Atomically loads a word starting at this address with the specified
  /// memory ordering.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicLoad(ordering: AtomicLoadOrdering) -> _AtomicInt16Storage {
    switch ordering {
    case .relaxed:
      return _AtomicInt16Storage(Builtin.atomicload_monotonic_Int16(_rawValue))
    case .acquiring:
      return _AtomicInt16Storage(Builtin.atomicload_acquire_Int16(_rawValue))
    case .sequentiallyConsistent:
      return _AtomicInt16Storage(Builtin.atomicload_seqcst_Int16(_rawValue))
    default:
      fatalError("Unsupported ordering")
    }
  }

  /// Atomically stores the specified value starting at the memory referenced by
  /// this pointer, with the specified memory ordering.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicStore(
    _ desired: _AtomicInt16Storage,
    ordering: AtomicStoreOrdering
  ) {
    switch ordering {
    case .relaxed:
      Builtin.atomicstore_monotonic_Int16(_rawValue, desired._value)
    case .releasing:
      Builtin.atomicstore_release_Int16(_rawValue, desired._value)
    case .sequentiallyConsistent:
      Builtin.atomicstore_seqcst_Int16(_rawValue, desired._value)
    default:
      fatalError("Unsupported ordering")
    }
  }

  /// Atomically stores the specified value starting at the memory referenced by
  /// this pointer, with the specified memory ordering.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicExchange(
    _ desired: _AtomicInt16Storage,
    ordering: AtomicUpdateOrdering
  ) -> _AtomicInt16Storage {
    switch ordering {
    case .relaxed:
      let oldValue = Builtin.atomicrmw_xchg_monotonic_Int16(
        _rawValue, desired._value)
      return _AtomicInt16Storage(oldValue)
    case .acquiring:
      let oldValue = Builtin.atomicrmw_xchg_acquire_Int16(
        _rawValue, desired._value)
      return _AtomicInt16Storage(oldValue)
    case .releasing:
      let oldValue = Builtin.atomicrmw_xchg_release_Int16(
        _rawValue, desired._value)
      return _AtomicInt16Storage(oldValue)
    case .acquiringAndReleasing:
      let oldValue = Builtin.atomicrmw_xchg_acqrel_Int16(
        _rawValue, desired._value)
      return _AtomicInt16Storage(oldValue)
    case .sequentiallyConsistent:
      let oldValue = Builtin.atomicrmw_xchg_seqcst_Int16(
        _rawValue, desired._value)
      return _AtomicInt16Storage(oldValue)
    default:
      fatalError("Unsupported ordering")
    }
  }

  /// Perform an atomic compare and exchange operation with the specified memory
  /// ordering.
  ///
  /// This operation is equivalent to the following pseudocode:
  ///
  /// ```
  /// atomic(self, ordering) { currentValue in
  ///   let original = currentValue
  ///   guard original == expected else { return (false, original) }
  ///   currentValue = desired
  ///   return (true, original)
  /// }
  /// ```
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicCompareExchange(
    expected: _AtomicInt16Storage,
    desired: _AtomicInt16Storage,
    ordering: AtomicUpdateOrdering
  ) -> (exchanged: Bool, original: _AtomicInt16Storage) {
    switch ordering {
    case .relaxed:
      let (oldValue, won) = Builtin.cmpxchg_monotonic_monotonic_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    case .acquiring:
      let (oldValue, won) = Builtin.cmpxchg_acquire_acquire_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    case .releasing:
      let (oldValue, won) = Builtin.cmpxchg_release_monotonic_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    case .acquiringAndReleasing:
      let (oldValue, won) = Builtin.cmpxchg_acqrel_acquire_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    case .sequentiallyConsistent:
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    default:
      fatalError("Unsupported ordering")
    }
  }

  /// Perform an atomic compare and exchange operation with the specified
  /// success/failure memory orderings.
  ///
  /// This operation is equivalent to the following pseudocode:
  ///
  /// ```
  /// atomic(self, ordering, failureOrdering) { currentValue in
  ///   let original = currentValue
  ///   guard original == expected else { return (false, original) }
  ///   currentValue = desired
  ///   return (true, original)
  /// }
  /// ```
  ///
  /// The `ordering` argument specifies the memory ordering to use when the
  /// operation manages to update the current value, while `failureOrdering`
  /// will be used when the operation leaves the value intact.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicCompareExchange(
    expected: _AtomicInt16Storage,
    desired: _AtomicInt16Storage,
    successOrdering: AtomicUpdateOrdering,
    failureOrdering: AtomicLoadOrdering
  ) -> (exchanged: Bool, original: _AtomicInt16Storage) {
    // FIXME: LLVM doesn't support arbitrary ordering combinations
    // yet, so upgrade the success ordering when necessary so that it
    // is at least as "strong" as the failure case.
    switch (successOrdering, failureOrdering) {
    case (.relaxed, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_monotonic_monotonic_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    case (.relaxed, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acquire_acquire_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    case (.relaxed, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    case (.acquiring, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_acquire_monotonic_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    case (.acquiring, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acquire_acquire_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    case (.acquiring, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    case (.releasing, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_release_monotonic_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    case (.releasing, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acqrel_acquire_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    case (.releasing, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    case (.acquiringAndReleasing, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_acqrel_monotonic_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    case (.acquiringAndReleasing, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acqrel_acquire_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    case (.acquiringAndReleasing, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    case (.sequentiallyConsistent, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_monotonic_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    case (.sequentiallyConsistent, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_acquire_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    case (.sequentiallyConsistent, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    default:
      preconditionFailure("Unsupported orderings")
    }
  }

  /// Perform an atomic compare and exchange operation with the specified
  /// success/failure memory orderings.
  ///
  /// This operation is equivalent to the following pseudocode:
  ///
  /// ```
  /// atomic(self, ordering, failureOrdering) { currentValue in
  ///   let original = currentValue
  ///   guard original == expected else { return (false, original) }
  ///   currentValue = desired
  ///   return (true, original)
  /// }
  /// ```
  ///
  /// The `ordering` argument specifies the memory ordering to use when the
  /// operation manages to update the current value, while `failureOrdering`
  /// will be used when the operation leaves the value intact.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicWeakCompareExchange(
    expected: _AtomicInt16Storage,
    desired: _AtomicInt16Storage,
    successOrdering: AtomicUpdateOrdering,
    failureOrdering: AtomicLoadOrdering
  ) -> (exchanged: Bool, original: _AtomicInt16Storage) {
    // FIXME: LLVM doesn't support arbitrary ordering combinations
    // yet, so upgrade the success ordering when necessary so that it
    // is at least as "strong" as the failure case.
    switch (successOrdering, failureOrdering) {
    case (.relaxed, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_monotonic_monotonic_weak_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    case (.relaxed, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acquire_acquire_weak_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    case (.relaxed, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_weak_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    case (.acquiring, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_acquire_monotonic_weak_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    case (.acquiring, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acquire_acquire_weak_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    case (.acquiring, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_weak_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    case (.releasing, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_release_monotonic_weak_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    case (.releasing, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acqrel_acquire_weak_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    case (.releasing, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_weak_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    case (.acquiringAndReleasing, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_acqrel_monotonic_weak_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    case (.acquiringAndReleasing, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acqrel_acquire_weak_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    case (.acquiringAndReleasing, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_weak_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    case (.sequentiallyConsistent, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_monotonic_weak_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    case (.sequentiallyConsistent, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_acquire_weak_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    case (.sequentiallyConsistent, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_weak_Int16(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt16Storage(oldValue))
    default:
      preconditionFailure("Unsupported orderings")
    }
  }

  /// Perform an atomic wrapping add operation and return the new value,
  /// with the specified memory ordering.
  ///
  /// - Note: This operation silently wraps around on overflow, like the
  /// `&+` operator does on `UInt` values.
  ///
  /// - Returns: The original value before the operation.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicLoadThenWrappingIncrement(
    by operand: _AtomicInt16Storage,
    ordering: AtomicUpdateOrdering
  ) -> _AtomicInt16Storage {
    switch ordering {
    case .relaxed:
      let value = Builtin.atomicrmw_add_monotonic_Int16(
        _rawValue, operand._value)
      return _AtomicInt16Storage(value)
    case .acquiring:
      let value = Builtin.atomicrmw_add_acquire_Int16(
        _rawValue, operand._value)
      return _AtomicInt16Storage(value)
    case .releasing:
      let value = Builtin.atomicrmw_add_release_Int16(
        _rawValue, operand._value)
      return _AtomicInt16Storage(value)
    case .acquiringAndReleasing:
      let value = Builtin.atomicrmw_add_acqrel_Int16(
        _rawValue, operand._value)
      return _AtomicInt16Storage(value)
    case .sequentiallyConsistent:
      let value = Builtin.atomicrmw_add_seqcst_Int16(
        _rawValue, operand._value)
      return _AtomicInt16Storage(value)
    default:
      preconditionFailure("Unsupported ordering")
    }
  }

  /// Perform an atomic wrapping subtract operation and return the new value,
  /// with the specified memory ordering.
  ///
  /// - Note: This operation silently wraps around on overflow, like the
  /// `&-` operator does on `UInt` values.
  ///
  /// - Returns: The original value before the operation.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicLoadThenWrappingDecrement(
    by operand: _AtomicInt16Storage,
    ordering: AtomicUpdateOrdering
  ) -> _AtomicInt16Storage {
    switch ordering {
    case .relaxed:
      let value = Builtin.atomicrmw_sub_monotonic_Int16(
        _rawValue, operand._value)
      return _AtomicInt16Storage(value)
    case .acquiring:
      let value = Builtin.atomicrmw_sub_acquire_Int16(
        _rawValue, operand._value)
      return _AtomicInt16Storage(value)
    case .releasing:
      let value = Builtin.atomicrmw_sub_release_Int16(
        _rawValue, operand._value)
      return _AtomicInt16Storage(value)
    case .acquiringAndReleasing:
      let value = Builtin.atomicrmw_sub_acqrel_Int16(
        _rawValue, operand._value)
      return _AtomicInt16Storage(value)
    case .sequentiallyConsistent:
      let value = Builtin.atomicrmw_sub_seqcst_Int16(
        _rawValue, operand._value)
      return _AtomicInt16Storage(value)
    default:
      preconditionFailure("Unsupported ordering")
    }
  }

  /// Perform an atomic bitwise AND operation and return the new value,
  /// with the specified memory ordering.
  ///
  /// - Returns: The original value before the operation.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicLoadThenBitwiseAnd(
    with operand: _AtomicInt16Storage,
    ordering: AtomicUpdateOrdering
  ) -> _AtomicInt16Storage {
    switch ordering {
    case .relaxed:
      let value = Builtin.atomicrmw_and_monotonic_Int16(
        _rawValue, operand._value)
      return _AtomicInt16Storage(value)
    case .acquiring:
      let value = Builtin.atomicrmw_and_acquire_Int16(
        _rawValue, operand._value)
      return _AtomicInt16Storage(value)
    case .releasing:
      let value = Builtin.atomicrmw_and_release_Int16(
        _rawValue, operand._value)
      return _AtomicInt16Storage(value)
    case .acquiringAndReleasing:
      let value = Builtin.atomicrmw_and_acqrel_Int16(
        _rawValue, operand._value)
      return _AtomicInt16Storage(value)
    case .sequentiallyConsistent:
      let value = Builtin.atomicrmw_and_seqcst_Int16(
        _rawValue, operand._value)
      return _AtomicInt16Storage(value)
    default:
      preconditionFailure("Unsupported ordering")
    }
  }

  /// Perform an atomic bitwise OR operation and return the new value,
  /// with the specified memory ordering.
  ///
  /// - Returns: The original value before the operation.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicLoadThenBitwiseOr(
    with operand: _AtomicInt16Storage,
    ordering: AtomicUpdateOrdering
  ) -> _AtomicInt16Storage {
    switch ordering {
    case .relaxed:
      let value = Builtin.atomicrmw_or_monotonic_Int16(
        _rawValue, operand._value)
      return _AtomicInt16Storage(value)
    case .acquiring:
      let value = Builtin.atomicrmw_or_acquire_Int16(
        _rawValue, operand._value)
      return _AtomicInt16Storage(value)
    case .releasing:
      let value = Builtin.atomicrmw_or_release_Int16(
        _rawValue, operand._value)
      return _AtomicInt16Storage(value)
    case .acquiringAndReleasing:
      let value = Builtin.atomicrmw_or_acqrel_Int16(
        _rawValue, operand._value)
      return _AtomicInt16Storage(value)
    case .sequentiallyConsistent:
      let value = Builtin.atomicrmw_or_seqcst_Int16(
        _rawValue, operand._value)
      return _AtomicInt16Storage(value)
    default:
      preconditionFailure("Unsupported ordering")
    }
  }

  /// Perform an atomic bitwise XOR operation and return the new value,
  /// with the specified memory ordering.
  ///
  /// - Returns: The original value before the operation.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicLoadThenBitwiseXor(
    with operand: _AtomicInt16Storage,
    ordering: AtomicUpdateOrdering
  ) -> _AtomicInt16Storage {
    switch ordering {
    case .relaxed:
      let value = Builtin.atomicrmw_xor_monotonic_Int16(
        _rawValue, operand._value)
      return _AtomicInt16Storage(value)
    case .acquiring:
      let value = Builtin.atomicrmw_xor_acquire_Int16(
        _rawValue, operand._value)
      return _AtomicInt16Storage(value)
    case .releasing:
      let value = Builtin.atomicrmw_xor_release_Int16(
        _rawValue, operand._value)
      return _AtomicInt16Storage(value)
    case .acquiringAndReleasing:
      let value = Builtin.atomicrmw_xor_acqrel_Int16(
        _rawValue, operand._value)
      return _AtomicInt16Storage(value)
    case .sequentiallyConsistent:
      let value = Builtin.atomicrmw_xor_seqcst_Int16(
        _rawValue, operand._value)
      return _AtomicInt16Storage(value)
    default:
      preconditionFailure("Unsupported ordering")
    }
  }
}

@usableFromInline
@frozen
@_alignment(4)
internal struct _AtomicInt32Storage {
  @usableFromInline
  internal var _value: Builtin.Int32

  @_alwaysEmitIntoClient @_transparent
  internal init(_ value: Builtin.Int32) {
    self._value = value
  }
}

extension UnsafeMutablePointer where Pointee == _AtomicInt32Storage {
  /// Atomically loads a word starting at this address with the specified
  /// memory ordering.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicLoad(ordering: AtomicLoadOrdering) -> _AtomicInt32Storage {
    switch ordering {
    case .relaxed:
      return _AtomicInt32Storage(Builtin.atomicload_monotonic_Int32(_rawValue))
    case .acquiring:
      return _AtomicInt32Storage(Builtin.atomicload_acquire_Int32(_rawValue))
    case .sequentiallyConsistent:
      return _AtomicInt32Storage(Builtin.atomicload_seqcst_Int32(_rawValue))
    default:
      fatalError("Unsupported ordering")
    }
  }

  /// Atomically stores the specified value starting at the memory referenced by
  /// this pointer, with the specified memory ordering.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicStore(
    _ desired: _AtomicInt32Storage,
    ordering: AtomicStoreOrdering
  ) {
    switch ordering {
    case .relaxed:
      Builtin.atomicstore_monotonic_Int32(_rawValue, desired._value)
    case .releasing:
      Builtin.atomicstore_release_Int32(_rawValue, desired._value)
    case .sequentiallyConsistent:
      Builtin.atomicstore_seqcst_Int32(_rawValue, desired._value)
    default:
      fatalError("Unsupported ordering")
    }
  }

  /// Atomically stores the specified value starting at the memory referenced by
  /// this pointer, with the specified memory ordering.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicExchange(
    _ desired: _AtomicInt32Storage,
    ordering: AtomicUpdateOrdering
  ) -> _AtomicInt32Storage {
    switch ordering {
    case .relaxed:
      let oldValue = Builtin.atomicrmw_xchg_monotonic_Int32(
        _rawValue, desired._value)
      return _AtomicInt32Storage(oldValue)
    case .acquiring:
      let oldValue = Builtin.atomicrmw_xchg_acquire_Int32(
        _rawValue, desired._value)
      return _AtomicInt32Storage(oldValue)
    case .releasing:
      let oldValue = Builtin.atomicrmw_xchg_release_Int32(
        _rawValue, desired._value)
      return _AtomicInt32Storage(oldValue)
    case .acquiringAndReleasing:
      let oldValue = Builtin.atomicrmw_xchg_acqrel_Int32(
        _rawValue, desired._value)
      return _AtomicInt32Storage(oldValue)
    case .sequentiallyConsistent:
      let oldValue = Builtin.atomicrmw_xchg_seqcst_Int32(
        _rawValue, desired._value)
      return _AtomicInt32Storage(oldValue)
    default:
      fatalError("Unsupported ordering")
    }
  }

  /// Perform an atomic compare and exchange operation with the specified memory
  /// ordering.
  ///
  /// This operation is equivalent to the following pseudocode:
  ///
  /// ```
  /// atomic(self, ordering) { currentValue in
  ///   let original = currentValue
  ///   guard original == expected else { return (false, original) }
  ///   currentValue = desired
  ///   return (true, original)
  /// }
  /// ```
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicCompareExchange(
    expected: _AtomicInt32Storage,
    desired: _AtomicInt32Storage,
    ordering: AtomicUpdateOrdering
  ) -> (exchanged: Bool, original: _AtomicInt32Storage) {
    switch ordering {
    case .relaxed:
      let (oldValue, won) = Builtin.cmpxchg_monotonic_monotonic_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    case .acquiring:
      let (oldValue, won) = Builtin.cmpxchg_acquire_acquire_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    case .releasing:
      let (oldValue, won) = Builtin.cmpxchg_release_monotonic_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    case .acquiringAndReleasing:
      let (oldValue, won) = Builtin.cmpxchg_acqrel_acquire_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    case .sequentiallyConsistent:
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    default:
      fatalError("Unsupported ordering")
    }
  }

  /// Perform an atomic compare and exchange operation with the specified
  /// success/failure memory orderings.
  ///
  /// This operation is equivalent to the following pseudocode:
  ///
  /// ```
  /// atomic(self, ordering, failureOrdering) { currentValue in
  ///   let original = currentValue
  ///   guard original == expected else { return (false, original) }
  ///   currentValue = desired
  ///   return (true, original)
  /// }
  /// ```
  ///
  /// The `ordering` argument specifies the memory ordering to use when the
  /// operation manages to update the current value, while `failureOrdering`
  /// will be used when the operation leaves the value intact.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicCompareExchange(
    expected: _AtomicInt32Storage,
    desired: _AtomicInt32Storage,
    successOrdering: AtomicUpdateOrdering,
    failureOrdering: AtomicLoadOrdering
  ) -> (exchanged: Bool, original: _AtomicInt32Storage) {
    // FIXME: LLVM doesn't support arbitrary ordering combinations
    // yet, so upgrade the success ordering when necessary so that it
    // is at least as "strong" as the failure case.
    switch (successOrdering, failureOrdering) {
    case (.relaxed, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_monotonic_monotonic_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    case (.relaxed, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acquire_acquire_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    case (.relaxed, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    case (.acquiring, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_acquire_monotonic_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    case (.acquiring, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acquire_acquire_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    case (.acquiring, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    case (.releasing, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_release_monotonic_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    case (.releasing, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acqrel_acquire_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    case (.releasing, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    case (.acquiringAndReleasing, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_acqrel_monotonic_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    case (.acquiringAndReleasing, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acqrel_acquire_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    case (.acquiringAndReleasing, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    case (.sequentiallyConsistent, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_monotonic_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    case (.sequentiallyConsistent, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_acquire_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    case (.sequentiallyConsistent, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    default:
      preconditionFailure("Unsupported orderings")
    }
  }

  /// Perform an atomic compare and exchange operation with the specified
  /// success/failure memory orderings.
  ///
  /// This operation is equivalent to the following pseudocode:
  ///
  /// ```
  /// atomic(self, ordering, failureOrdering) { currentValue in
  ///   let original = currentValue
  ///   guard original == expected else { return (false, original) }
  ///   currentValue = desired
  ///   return (true, original)
  /// }
  /// ```
  ///
  /// The `ordering` argument specifies the memory ordering to use when the
  /// operation manages to update the current value, while `failureOrdering`
  /// will be used when the operation leaves the value intact.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicWeakCompareExchange(
    expected: _AtomicInt32Storage,
    desired: _AtomicInt32Storage,
    successOrdering: AtomicUpdateOrdering,
    failureOrdering: AtomicLoadOrdering
  ) -> (exchanged: Bool, original: _AtomicInt32Storage) {
    // FIXME: LLVM doesn't support arbitrary ordering combinations
    // yet, so upgrade the success ordering when necessary so that it
    // is at least as "strong" as the failure case.
    switch (successOrdering, failureOrdering) {
    case (.relaxed, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_monotonic_monotonic_weak_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    case (.relaxed, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acquire_acquire_weak_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    case (.relaxed, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_weak_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    case (.acquiring, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_acquire_monotonic_weak_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    case (.acquiring, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acquire_acquire_weak_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    case (.acquiring, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_weak_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    case (.releasing, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_release_monotonic_weak_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    case (.releasing, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acqrel_acquire_weak_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    case (.releasing, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_weak_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    case (.acquiringAndReleasing, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_acqrel_monotonic_weak_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    case (.acquiringAndReleasing, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acqrel_acquire_weak_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    case (.acquiringAndReleasing, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_weak_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    case (.sequentiallyConsistent, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_monotonic_weak_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    case (.sequentiallyConsistent, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_acquire_weak_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    case (.sequentiallyConsistent, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_weak_Int32(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt32Storage(oldValue))
    default:
      preconditionFailure("Unsupported orderings")
    }
  }

  /// Perform an atomic wrapping add operation and return the new value,
  /// with the specified memory ordering.
  ///
  /// - Note: This operation silently wraps around on overflow, like the
  /// `&+` operator does on `UInt` values.
  ///
  /// - Returns: The original value before the operation.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicLoadThenWrappingIncrement(
    by operand: _AtomicInt32Storage,
    ordering: AtomicUpdateOrdering
  ) -> _AtomicInt32Storage {
    switch ordering {
    case .relaxed:
      let value = Builtin.atomicrmw_add_monotonic_Int32(
        _rawValue, operand._value)
      return _AtomicInt32Storage(value)
    case .acquiring:
      let value = Builtin.atomicrmw_add_acquire_Int32(
        _rawValue, operand._value)
      return _AtomicInt32Storage(value)
    case .releasing:
      let value = Builtin.atomicrmw_add_release_Int32(
        _rawValue, operand._value)
      return _AtomicInt32Storage(value)
    case .acquiringAndReleasing:
      let value = Builtin.atomicrmw_add_acqrel_Int32(
        _rawValue, operand._value)
      return _AtomicInt32Storage(value)
    case .sequentiallyConsistent:
      let value = Builtin.atomicrmw_add_seqcst_Int32(
        _rawValue, operand._value)
      return _AtomicInt32Storage(value)
    default:
      preconditionFailure("Unsupported ordering")
    }
  }

  /// Perform an atomic wrapping subtract operation and return the new value,
  /// with the specified memory ordering.
  ///
  /// - Note: This operation silently wraps around on overflow, like the
  /// `&-` operator does on `UInt` values.
  ///
  /// - Returns: The original value before the operation.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicLoadThenWrappingDecrement(
    by operand: _AtomicInt32Storage,
    ordering: AtomicUpdateOrdering
  ) -> _AtomicInt32Storage {
    switch ordering {
    case .relaxed:
      let value = Builtin.atomicrmw_sub_monotonic_Int32(
        _rawValue, operand._value)
      return _AtomicInt32Storage(value)
    case .acquiring:
      let value = Builtin.atomicrmw_sub_acquire_Int32(
        _rawValue, operand._value)
      return _AtomicInt32Storage(value)
    case .releasing:
      let value = Builtin.atomicrmw_sub_release_Int32(
        _rawValue, operand._value)
      return _AtomicInt32Storage(value)
    case .acquiringAndReleasing:
      let value = Builtin.atomicrmw_sub_acqrel_Int32(
        _rawValue, operand._value)
      return _AtomicInt32Storage(value)
    case .sequentiallyConsistent:
      let value = Builtin.atomicrmw_sub_seqcst_Int32(
        _rawValue, operand._value)
      return _AtomicInt32Storage(value)
    default:
      preconditionFailure("Unsupported ordering")
    }
  }

  /// Perform an atomic bitwise AND operation and return the new value,
  /// with the specified memory ordering.
  ///
  /// - Returns: The original value before the operation.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicLoadThenBitwiseAnd(
    with operand: _AtomicInt32Storage,
    ordering: AtomicUpdateOrdering
  ) -> _AtomicInt32Storage {
    switch ordering {
    case .relaxed:
      let value = Builtin.atomicrmw_and_monotonic_Int32(
        _rawValue, operand._value)
      return _AtomicInt32Storage(value)
    case .acquiring:
      let value = Builtin.atomicrmw_and_acquire_Int32(
        _rawValue, operand._value)
      return _AtomicInt32Storage(value)
    case .releasing:
      let value = Builtin.atomicrmw_and_release_Int32(
        _rawValue, operand._value)
      return _AtomicInt32Storage(value)
    case .acquiringAndReleasing:
      let value = Builtin.atomicrmw_and_acqrel_Int32(
        _rawValue, operand._value)
      return _AtomicInt32Storage(value)
    case .sequentiallyConsistent:
      let value = Builtin.atomicrmw_and_seqcst_Int32(
        _rawValue, operand._value)
      return _AtomicInt32Storage(value)
    default:
      preconditionFailure("Unsupported ordering")
    }
  }

  /// Perform an atomic bitwise OR operation and return the new value,
  /// with the specified memory ordering.
  ///
  /// - Returns: The original value before the operation.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicLoadThenBitwiseOr(
    with operand: _AtomicInt32Storage,
    ordering: AtomicUpdateOrdering
  ) -> _AtomicInt32Storage {
    switch ordering {
    case .relaxed:
      let value = Builtin.atomicrmw_or_monotonic_Int32(
        _rawValue, operand._value)
      return _AtomicInt32Storage(value)
    case .acquiring:
      let value = Builtin.atomicrmw_or_acquire_Int32(
        _rawValue, operand._value)
      return _AtomicInt32Storage(value)
    case .releasing:
      let value = Builtin.atomicrmw_or_release_Int32(
        _rawValue, operand._value)
      return _AtomicInt32Storage(value)
    case .acquiringAndReleasing:
      let value = Builtin.atomicrmw_or_acqrel_Int32(
        _rawValue, operand._value)
      return _AtomicInt32Storage(value)
    case .sequentiallyConsistent:
      let value = Builtin.atomicrmw_or_seqcst_Int32(
        _rawValue, operand._value)
      return _AtomicInt32Storage(value)
    default:
      preconditionFailure("Unsupported ordering")
    }
  }

  /// Perform an atomic bitwise XOR operation and return the new value,
  /// with the specified memory ordering.
  ///
  /// - Returns: The original value before the operation.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicLoadThenBitwiseXor(
    with operand: _AtomicInt32Storage,
    ordering: AtomicUpdateOrdering
  ) -> _AtomicInt32Storage {
    switch ordering {
    case .relaxed:
      let value = Builtin.atomicrmw_xor_monotonic_Int32(
        _rawValue, operand._value)
      return _AtomicInt32Storage(value)
    case .acquiring:
      let value = Builtin.atomicrmw_xor_acquire_Int32(
        _rawValue, operand._value)
      return _AtomicInt32Storage(value)
    case .releasing:
      let value = Builtin.atomicrmw_xor_release_Int32(
        _rawValue, operand._value)
      return _AtomicInt32Storage(value)
    case .acquiringAndReleasing:
      let value = Builtin.atomicrmw_xor_acqrel_Int32(
        _rawValue, operand._value)
      return _AtomicInt32Storage(value)
    case .sequentiallyConsistent:
      let value = Builtin.atomicrmw_xor_seqcst_Int32(
        _rawValue, operand._value)
      return _AtomicInt32Storage(value)
    default:
      preconditionFailure("Unsupported ordering")
    }
  }
}

@usableFromInline
@frozen
@_alignment(8)
internal struct _AtomicInt64Storage {
  @usableFromInline
  internal var _value: Builtin.Int64

  @_alwaysEmitIntoClient @_transparent
  internal init(_ value: Builtin.Int64) {
    self._value = value
  }
}

extension UnsafeMutablePointer where Pointee == _AtomicInt64Storage {
  /// Atomically loads a word starting at this address with the specified
  /// memory ordering.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicLoad(ordering: AtomicLoadOrdering) -> _AtomicInt64Storage {
    switch ordering {
    case .relaxed:
      return _AtomicInt64Storage(Builtin.atomicload_monotonic_Int64(_rawValue))
    case .acquiring:
      return _AtomicInt64Storage(Builtin.atomicload_acquire_Int64(_rawValue))
    case .sequentiallyConsistent:
      return _AtomicInt64Storage(Builtin.atomicload_seqcst_Int64(_rawValue))
    default:
      fatalError("Unsupported ordering")
    }
  }

  /// Atomically stores the specified value starting at the memory referenced by
  /// this pointer, with the specified memory ordering.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicStore(
    _ desired: _AtomicInt64Storage,
    ordering: AtomicStoreOrdering
  ) {
    switch ordering {
    case .relaxed:
      Builtin.atomicstore_monotonic_Int64(_rawValue, desired._value)
    case .releasing:
      Builtin.atomicstore_release_Int64(_rawValue, desired._value)
    case .sequentiallyConsistent:
      Builtin.atomicstore_seqcst_Int64(_rawValue, desired._value)
    default:
      fatalError("Unsupported ordering")
    }
  }

  /// Atomically stores the specified value starting at the memory referenced by
  /// this pointer, with the specified memory ordering.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicExchange(
    _ desired: _AtomicInt64Storage,
    ordering: AtomicUpdateOrdering
  ) -> _AtomicInt64Storage {
    switch ordering {
    case .relaxed:
      let oldValue = Builtin.atomicrmw_xchg_monotonic_Int64(
        _rawValue, desired._value)
      return _AtomicInt64Storage(oldValue)
    case .acquiring:
      let oldValue = Builtin.atomicrmw_xchg_acquire_Int64(
        _rawValue, desired._value)
      return _AtomicInt64Storage(oldValue)
    case .releasing:
      let oldValue = Builtin.atomicrmw_xchg_release_Int64(
        _rawValue, desired._value)
      return _AtomicInt64Storage(oldValue)
    case .acquiringAndReleasing:
      let oldValue = Builtin.atomicrmw_xchg_acqrel_Int64(
        _rawValue, desired._value)
      return _AtomicInt64Storage(oldValue)
    case .sequentiallyConsistent:
      let oldValue = Builtin.atomicrmw_xchg_seqcst_Int64(
        _rawValue, desired._value)
      return _AtomicInt64Storage(oldValue)
    default:
      fatalError("Unsupported ordering")
    }
  }

  /// Perform an atomic compare and exchange operation with the specified memory
  /// ordering.
  ///
  /// This operation is equivalent to the following pseudocode:
  ///
  /// ```
  /// atomic(self, ordering) { currentValue in
  ///   let original = currentValue
  ///   guard original == expected else { return (false, original) }
  ///   currentValue = desired
  ///   return (true, original)
  /// }
  /// ```
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicCompareExchange(
    expected: _AtomicInt64Storage,
    desired: _AtomicInt64Storage,
    ordering: AtomicUpdateOrdering
  ) -> (exchanged: Bool, original: _AtomicInt64Storage) {
    switch ordering {
    case .relaxed:
      let (oldValue, won) = Builtin.cmpxchg_monotonic_monotonic_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    case .acquiring:
      let (oldValue, won) = Builtin.cmpxchg_acquire_acquire_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    case .releasing:
      let (oldValue, won) = Builtin.cmpxchg_release_monotonic_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    case .acquiringAndReleasing:
      let (oldValue, won) = Builtin.cmpxchg_acqrel_acquire_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    case .sequentiallyConsistent:
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    default:
      fatalError("Unsupported ordering")
    }
  }

  /// Perform an atomic compare and exchange operation with the specified
  /// success/failure memory orderings.
  ///
  /// This operation is equivalent to the following pseudocode:
  ///
  /// ```
  /// atomic(self, ordering, failureOrdering) { currentValue in
  ///   let original = currentValue
  ///   guard original == expected else { return (false, original) }
  ///   currentValue = desired
  ///   return (true, original)
  /// }
  /// ```
  ///
  /// The `ordering` argument specifies the memory ordering to use when the
  /// operation manages to update the current value, while `failureOrdering`
  /// will be used when the operation leaves the value intact.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicCompareExchange(
    expected: _AtomicInt64Storage,
    desired: _AtomicInt64Storage,
    successOrdering: AtomicUpdateOrdering,
    failureOrdering: AtomicLoadOrdering
  ) -> (exchanged: Bool, original: _AtomicInt64Storage) {
    // FIXME: LLVM doesn't support arbitrary ordering combinations
    // yet, so upgrade the success ordering when necessary so that it
    // is at least as "strong" as the failure case.
    switch (successOrdering, failureOrdering) {
    case (.relaxed, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_monotonic_monotonic_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    case (.relaxed, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acquire_acquire_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    case (.relaxed, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    case (.acquiring, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_acquire_monotonic_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    case (.acquiring, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acquire_acquire_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    case (.acquiring, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    case (.releasing, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_release_monotonic_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    case (.releasing, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acqrel_acquire_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    case (.releasing, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    case (.acquiringAndReleasing, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_acqrel_monotonic_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    case (.acquiringAndReleasing, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acqrel_acquire_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    case (.acquiringAndReleasing, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    case (.sequentiallyConsistent, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_monotonic_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    case (.sequentiallyConsistent, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_acquire_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    case (.sequentiallyConsistent, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    default:
      preconditionFailure("Unsupported orderings")
    }
  }

  /// Perform an atomic compare and exchange operation with the specified
  /// success/failure memory orderings.
  ///
  /// This operation is equivalent to the following pseudocode:
  ///
  /// ```
  /// atomic(self, ordering, failureOrdering) { currentValue in
  ///   let original = currentValue
  ///   guard original == expected else { return (false, original) }
  ///   currentValue = desired
  ///   return (true, original)
  /// }
  /// ```
  ///
  /// The `ordering` argument specifies the memory ordering to use when the
  /// operation manages to update the current value, while `failureOrdering`
  /// will be used when the operation leaves the value intact.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicWeakCompareExchange(
    expected: _AtomicInt64Storage,
    desired: _AtomicInt64Storage,
    successOrdering: AtomicUpdateOrdering,
    failureOrdering: AtomicLoadOrdering
  ) -> (exchanged: Bool, original: _AtomicInt64Storage) {
    // FIXME: LLVM doesn't support arbitrary ordering combinations
    // yet, so upgrade the success ordering when necessary so that it
    // is at least as "strong" as the failure case.
    switch (successOrdering, failureOrdering) {
    case (.relaxed, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_monotonic_monotonic_weak_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    case (.relaxed, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acquire_acquire_weak_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    case (.relaxed, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_weak_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    case (.acquiring, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_acquire_monotonic_weak_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    case (.acquiring, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acquire_acquire_weak_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    case (.acquiring, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_weak_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    case (.releasing, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_release_monotonic_weak_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    case (.releasing, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acqrel_acquire_weak_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    case (.releasing, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_weak_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    case (.acquiringAndReleasing, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_acqrel_monotonic_weak_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    case (.acquiringAndReleasing, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acqrel_acquire_weak_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    case (.acquiringAndReleasing, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_weak_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    case (.sequentiallyConsistent, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_monotonic_weak_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    case (.sequentiallyConsistent, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_acquire_weak_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    case (.sequentiallyConsistent, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_weak_Int64(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt64Storage(oldValue))
    default:
      preconditionFailure("Unsupported orderings")
    }
  }

  /// Perform an atomic wrapping add operation and return the new value,
  /// with the specified memory ordering.
  ///
  /// - Note: This operation silently wraps around on overflow, like the
  /// `&+` operator does on `UInt` values.
  ///
  /// - Returns: The original value before the operation.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicLoadThenWrappingIncrement(
    by operand: _AtomicInt64Storage,
    ordering: AtomicUpdateOrdering
  ) -> _AtomicInt64Storage {
    switch ordering {
    case .relaxed:
      let value = Builtin.atomicrmw_add_monotonic_Int64(
        _rawValue, operand._value)
      return _AtomicInt64Storage(value)
    case .acquiring:
      let value = Builtin.atomicrmw_add_acquire_Int64(
        _rawValue, operand._value)
      return _AtomicInt64Storage(value)
    case .releasing:
      let value = Builtin.atomicrmw_add_release_Int64(
        _rawValue, operand._value)
      return _AtomicInt64Storage(value)
    case .acquiringAndReleasing:
      let value = Builtin.atomicrmw_add_acqrel_Int64(
        _rawValue, operand._value)
      return _AtomicInt64Storage(value)
    case .sequentiallyConsistent:
      let value = Builtin.atomicrmw_add_seqcst_Int64(
        _rawValue, operand._value)
      return _AtomicInt64Storage(value)
    default:
      preconditionFailure("Unsupported ordering")
    }
  }

  /// Perform an atomic wrapping subtract operation and return the new value,
  /// with the specified memory ordering.
  ///
  /// - Note: This operation silently wraps around on overflow, like the
  /// `&-` operator does on `UInt` values.
  ///
  /// - Returns: The original value before the operation.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicLoadThenWrappingDecrement(
    by operand: _AtomicInt64Storage,
    ordering: AtomicUpdateOrdering
  ) -> _AtomicInt64Storage {
    switch ordering {
    case .relaxed:
      let value = Builtin.atomicrmw_sub_monotonic_Int64(
        _rawValue, operand._value)
      return _AtomicInt64Storage(value)
    case .acquiring:
      let value = Builtin.atomicrmw_sub_acquire_Int64(
        _rawValue, operand._value)
      return _AtomicInt64Storage(value)
    case .releasing:
      let value = Builtin.atomicrmw_sub_release_Int64(
        _rawValue, operand._value)
      return _AtomicInt64Storage(value)
    case .acquiringAndReleasing:
      let value = Builtin.atomicrmw_sub_acqrel_Int64(
        _rawValue, operand._value)
      return _AtomicInt64Storage(value)
    case .sequentiallyConsistent:
      let value = Builtin.atomicrmw_sub_seqcst_Int64(
        _rawValue, operand._value)
      return _AtomicInt64Storage(value)
    default:
      preconditionFailure("Unsupported ordering")
    }
  }

  /// Perform an atomic bitwise AND operation and return the new value,
  /// with the specified memory ordering.
  ///
  /// - Returns: The original value before the operation.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicLoadThenBitwiseAnd(
    with operand: _AtomicInt64Storage,
    ordering: AtomicUpdateOrdering
  ) -> _AtomicInt64Storage {
    switch ordering {
    case .relaxed:
      let value = Builtin.atomicrmw_and_monotonic_Int64(
        _rawValue, operand._value)
      return _AtomicInt64Storage(value)
    case .acquiring:
      let value = Builtin.atomicrmw_and_acquire_Int64(
        _rawValue, operand._value)
      return _AtomicInt64Storage(value)
    case .releasing:
      let value = Builtin.atomicrmw_and_release_Int64(
        _rawValue, operand._value)
      return _AtomicInt64Storage(value)
    case .acquiringAndReleasing:
      let value = Builtin.atomicrmw_and_acqrel_Int64(
        _rawValue, operand._value)
      return _AtomicInt64Storage(value)
    case .sequentiallyConsistent:
      let value = Builtin.atomicrmw_and_seqcst_Int64(
        _rawValue, operand._value)
      return _AtomicInt64Storage(value)
    default:
      preconditionFailure("Unsupported ordering")
    }
  }

  /// Perform an atomic bitwise OR operation and return the new value,
  /// with the specified memory ordering.
  ///
  /// - Returns: The original value before the operation.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicLoadThenBitwiseOr(
    with operand: _AtomicInt64Storage,
    ordering: AtomicUpdateOrdering
  ) -> _AtomicInt64Storage {
    switch ordering {
    case .relaxed:
      let value = Builtin.atomicrmw_or_monotonic_Int64(
        _rawValue, operand._value)
      return _AtomicInt64Storage(value)
    case .acquiring:
      let value = Builtin.atomicrmw_or_acquire_Int64(
        _rawValue, operand._value)
      return _AtomicInt64Storage(value)
    case .releasing:
      let value = Builtin.atomicrmw_or_release_Int64(
        _rawValue, operand._value)
      return _AtomicInt64Storage(value)
    case .acquiringAndReleasing:
      let value = Builtin.atomicrmw_or_acqrel_Int64(
        _rawValue, operand._value)
      return _AtomicInt64Storage(value)
    case .sequentiallyConsistent:
      let value = Builtin.atomicrmw_or_seqcst_Int64(
        _rawValue, operand._value)
      return _AtomicInt64Storage(value)
    default:
      preconditionFailure("Unsupported ordering")
    }
  }

  /// Perform an atomic bitwise XOR operation and return the new value,
  /// with the specified memory ordering.
  ///
  /// - Returns: The original value before the operation.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicLoadThenBitwiseXor(
    with operand: _AtomicInt64Storage,
    ordering: AtomicUpdateOrdering
  ) -> _AtomicInt64Storage {
    switch ordering {
    case .relaxed:
      let value = Builtin.atomicrmw_xor_monotonic_Int64(
        _rawValue, operand._value)
      return _AtomicInt64Storage(value)
    case .acquiring:
      let value = Builtin.atomicrmw_xor_acquire_Int64(
        _rawValue, operand._value)
      return _AtomicInt64Storage(value)
    case .releasing:
      let value = Builtin.atomicrmw_xor_release_Int64(
        _rawValue, operand._value)
      return _AtomicInt64Storage(value)
    case .acquiringAndReleasing:
      let value = Builtin.atomicrmw_xor_acqrel_Int64(
        _rawValue, operand._value)
      return _AtomicInt64Storage(value)
    case .sequentiallyConsistent:
      let value = Builtin.atomicrmw_xor_seqcst_Int64(
        _rawValue, operand._value)
      return _AtomicInt64Storage(value)
    default:
      preconditionFailure("Unsupported ordering")
    }
  }
}

#if _pointerBitWidth(_64)
@usableFromInline
@frozen
@_alignment(16)
internal struct _AtomicInt128Storage {
  @usableFromInline
  internal var _value: Builtin.Int128

  @_alwaysEmitIntoClient @_transparent
  internal init(_ value: Builtin.Int128) {
    self._value = value
  }
}

extension UnsafeMutablePointer where Pointee == _AtomicInt128Storage {
  /// Atomically loads a word starting at this address with the specified
  /// memory ordering.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicLoad(ordering: AtomicLoadOrdering) -> _AtomicInt128Storage {
    switch ordering {
    case .relaxed:
      return _AtomicInt128Storage(Builtin.atomicload_monotonic_Int128(_rawValue))
    case .acquiring:
      return _AtomicInt128Storage(Builtin.atomicload_acquire_Int128(_rawValue))
    case .sequentiallyConsistent:
      return _AtomicInt128Storage(Builtin.atomicload_seqcst_Int128(_rawValue))
    default:
      fatalError("Unsupported ordering")
    }
  }

  /// Atomically stores the specified value starting at the memory referenced by
  /// this pointer, with the specified memory ordering.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicStore(
    _ desired: _AtomicInt128Storage,
    ordering: AtomicStoreOrdering
  ) {
    switch ordering {
    case .relaxed:
      Builtin.atomicstore_monotonic_Int128(_rawValue, desired._value)
    case .releasing:
      Builtin.atomicstore_release_Int128(_rawValue, desired._value)
    case .sequentiallyConsistent:
      Builtin.atomicstore_seqcst_Int128(_rawValue, desired._value)
    default:
      fatalError("Unsupported ordering")
    }
  }

  /// Atomically stores the specified value starting at the memory referenced by
  /// this pointer, with the specified memory ordering.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicExchange(
    _ desired: _AtomicInt128Storage,
    ordering: AtomicUpdateOrdering
  ) -> _AtomicInt128Storage {
    switch ordering {
    case .relaxed:
      let oldValue = Builtin.atomicrmw_xchg_monotonic_Int128(
        _rawValue, desired._value)
      return _AtomicInt128Storage(oldValue)
    case .acquiring:
      let oldValue = Builtin.atomicrmw_xchg_acquire_Int128(
        _rawValue, desired._value)
      return _AtomicInt128Storage(oldValue)
    case .releasing:
      let oldValue = Builtin.atomicrmw_xchg_release_Int128(
        _rawValue, desired._value)
      return _AtomicInt128Storage(oldValue)
    case .acquiringAndReleasing:
      let oldValue = Builtin.atomicrmw_xchg_acqrel_Int128(
        _rawValue, desired._value)
      return _AtomicInt128Storage(oldValue)
    case .sequentiallyConsistent:
      let oldValue = Builtin.atomicrmw_xchg_seqcst_Int128(
        _rawValue, desired._value)
      return _AtomicInt128Storage(oldValue)
    default:
      fatalError("Unsupported ordering")
    }
  }

  /// Perform an atomic compare and exchange operation with the specified memory
  /// ordering.
  ///
  /// This operation is equivalent to the following pseudocode:
  ///
  /// ```
  /// atomic(self, ordering) { currentValue in
  ///   let original = currentValue
  ///   guard original == expected else { return (false, original) }
  ///   currentValue = desired
  ///   return (true, original)
  /// }
  /// ```
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicCompareExchange(
    expected: _AtomicInt128Storage,
    desired: _AtomicInt128Storage,
    ordering: AtomicUpdateOrdering
  ) -> (exchanged: Bool, original: _AtomicInt128Storage) {
    switch ordering {
    case .relaxed:
      let (oldValue, won) = Builtin.cmpxchg_monotonic_monotonic_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    case .acquiring:
      let (oldValue, won) = Builtin.cmpxchg_acquire_acquire_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    case .releasing:
      let (oldValue, won) = Builtin.cmpxchg_release_monotonic_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    case .acquiringAndReleasing:
      let (oldValue, won) = Builtin.cmpxchg_acqrel_acquire_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    case .sequentiallyConsistent:
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    default:
      fatalError("Unsupported ordering")
    }
  }

  /// Perform an atomic compare and exchange operation with the specified
  /// success/failure memory orderings.
  ///
  /// This operation is equivalent to the following pseudocode:
  ///
  /// ```
  /// atomic(self, ordering, failureOrdering) { currentValue in
  ///   let original = currentValue
  ///   guard original == expected else { return (false, original) }
  ///   currentValue = desired
  ///   return (true, original)
  /// }
  /// ```
  ///
  /// The `ordering` argument specifies the memory ordering to use when the
  /// operation manages to update the current value, while `failureOrdering`
  /// will be used when the operation leaves the value intact.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicCompareExchange(
    expected: _AtomicInt128Storage,
    desired: _AtomicInt128Storage,
    successOrdering: AtomicUpdateOrdering,
    failureOrdering: AtomicLoadOrdering
  ) -> (exchanged: Bool, original: _AtomicInt128Storage) {
    // FIXME: LLVM doesn't support arbitrary ordering combinations
    // yet, so upgrade the success ordering when necessary so that it
    // is at least as "strong" as the failure case.
    switch (successOrdering, failureOrdering) {
    case (.relaxed, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_monotonic_monotonic_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    case (.relaxed, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acquire_acquire_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    case (.relaxed, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    case (.acquiring, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_acquire_monotonic_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    case (.acquiring, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acquire_acquire_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    case (.acquiring, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    case (.releasing, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_release_monotonic_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    case (.releasing, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acqrel_acquire_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    case (.releasing, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    case (.acquiringAndReleasing, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_acqrel_monotonic_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    case (.acquiringAndReleasing, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acqrel_acquire_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    case (.acquiringAndReleasing, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    case (.sequentiallyConsistent, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_monotonic_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    case (.sequentiallyConsistent, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_acquire_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    case (.sequentiallyConsistent, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    default:
      preconditionFailure("Unsupported orderings")
    }
  }

  /// Perform an atomic compare and exchange operation with the specified
  /// success/failure memory orderings.
  ///
  /// This operation is equivalent to the following pseudocode:
  ///
  /// ```
  /// atomic(self, ordering, failureOrdering) { currentValue in
  ///   let original = currentValue
  ///   guard original == expected else { return (false, original) }
  ///   currentValue = desired
  ///   return (true, original)
  /// }
  /// ```
  ///
  /// The `ordering` argument specifies the memory ordering to use when the
  /// operation manages to update the current value, while `failureOrdering`
  /// will be used when the operation leaves the value intact.
  @_semantics("atomics.requires_constant_orderings")
  @_alwaysEmitIntoClient
  @_transparent // Debug performance
  internal func _atomicWeakCompareExchange(
    expected: _AtomicInt128Storage,
    desired: _AtomicInt128Storage,
    successOrdering: AtomicUpdateOrdering,
    failureOrdering: AtomicLoadOrdering
  ) -> (exchanged: Bool, original: _AtomicInt128Storage) {
    // FIXME: LLVM doesn't support arbitrary ordering combinations
    // yet, so upgrade the success ordering when necessary so that it
    // is at least as "strong" as the failure case.
    switch (successOrdering, failureOrdering) {
    case (.relaxed, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_monotonic_monotonic_weak_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    case (.relaxed, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acquire_acquire_weak_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    case (.relaxed, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_weak_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    case (.acquiring, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_acquire_monotonic_weak_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    case (.acquiring, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acquire_acquire_weak_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    case (.acquiring, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_weak_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    case (.releasing, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_release_monotonic_weak_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    case (.releasing, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acqrel_acquire_weak_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    case (.releasing, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_weak_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    case (.acquiringAndReleasing, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_acqrel_monotonic_weak_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    case (.acquiringAndReleasing, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_acqrel_acquire_weak_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    case (.acquiringAndReleasing, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_weak_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    case (.sequentiallyConsistent, .relaxed):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_monotonic_weak_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    case (.sequentiallyConsistent, .acquiring):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_acquire_weak_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    case (.sequentiallyConsistent, .sequentiallyConsistent):
      let (oldValue, won) = Builtin.cmpxchg_seqcst_seqcst_weak_Int128(
        _rawValue, expected._value, desired._value)
      return (Bool(won), _AtomicInt128Storage(oldValue))
    default:
      preconditionFailure("Unsupported orderings")
    }
  }
}
#endif

#if _pointerBitWidth(_64)
@usableFromInline internal typealias _AtomicIntStorage = _AtomicInt64Storage
@usableFromInline internal typealias _AtomicDoubleWordStorage = _AtomicInt128Storage
#elseif _pointerBitWidth(_32)
@usableFromInline internal typealias _AtomicIntStorage = _AtomicInt32Storage
@usableFromInline internal typealias _AtomicDoubleWordStorage = _AtomicInt64Storage
#else
#error("Unexpected pointer bit width")
#endif
